package com.tangpian.sna.core.fetcher;

import java.io.IOException;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Date;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Random;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import org.jsoup.Connection;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.stereotype.Component;

import com.alibaba.fastjson.JSON;
import com.tangpian.sna.core.dao.ItemDao;
import com.tangpian.sna.core.model.WeiboItem;

@Component
public class WeiboFetcher implements Fetcher {

	private static Logger logger = LoggerFactory.getLogger(WeiboFetcher.class);

	private ItemDao<WeiboItem> itemDao;
	
	public void setItemDao(ItemDao<WeiboItem> itemDao) {
		this.itemDao = itemDao;
	}

	private KeyWordsPolicy policy = new KeyWordsPolicy();
	private int maxPages = 1000; // 每天抓取的最大页数
	private int maxDays = 90;// 抓取的天数
	private Date date = new Date();

	public void setMaxDays(int maxDays) {
		this.maxDays = maxDays;
	}

	public void setMaxPages(int maxPages) {
		this.maxPages = maxPages;
	}

	public void setDate(Date date) {
		this.date = date;
	}

	// 文本块正文匹配正则
	private final String blockRegex = "<script>STK\\s&&\\sSTK\\.pageletM\\s&&\\sSTK\\.pageletM\\.view\\(.*\\)";
	private Pattern pattern = Pattern.compile(blockRegex);

	@Override
	public void fetch(String keyword) {

		for (int i = 0; i <= maxDays; i++) {
			for (int j = 0; j < 24; j++) {

				String startDate = new SimpleDateFormat("yyyy-MM-dd-H")
						.format(date.getTime() - i * 24 * 60 * 60 * 1000L
								- j * 60 * 60 * 1000L);
				String finishDate = new SimpleDateFormat("yyyy-MM-dd-H")
						.format(date.getTime() - i * 24 * 60 * 60 * 1000L - j
								* 60 * 60 * 1000L);
				fetchPagesByDate(keyword, startDate, finishDate);
			}
		}

	}

	private void fetchPagesByDate(String keyword, String startDate,
			String finishDate) {
		for (int i = 0; i < maxPages; i++) {
			List<WeiboItem> list = new ArrayList<WeiboItem>();
			int retryCount = 3;

			String page = "";

			if (i > 0) {
				page = "&page=" + (i + 1);
			}

			String date = "custom:" + startDate + ":" + finishDate;

			List<Element> elements = null;
			// 抓取返回50个内容
			try {
				logger.debug("fetch data at " + date + " page: " + (i + 1));

				elements = fetchPageElements(keyword, date, page);
			} catch (IOException e) {
//				e.printStackTrace();
			}

			while ((elements == null || elements.size() <= 0) && retryCount > 0) {
				try {
					elements = fetchPageElements(keyword, date, page);
				} catch (IOException e) {
					// TODO Auto-generated catch block
//					e.printStackTrace();
				}
				retryCount--;
				if (retryCount <= 0) {
					logger.debug("No more data to fetch with current keyword.");
					return;
				}
			}

			logger.debug("page size:" + elements.size());
			for (Element elem : elements) {

				String name = elem.select("p[node-type=feed_list_content]")
						.select("a").attr("nick-name");
				String forwordContent = elem
						.select("dt[node-type=feed_list_forwardContent]")
						.select("em").text();
				String content = elem.select("p[node-type=feed_list_content]")
						.select("em").text();
				String text = content + "//" + forwordContent;
				String id = elem.select("dl[class=feed_list]").attr("mid");
				String forwordCount = elem
						.select("p[class=info W_linkb W_textb]").select("a")
						.get(0).text();
				String colloctCount = elem
						.select("p[class=info W_linkb W_textb]").select("a")
						.get(1).text();
				String commentCount = elem
						.select("p[class=info W_linkb W_textb]").select("a")
						.get(2).text();
				// String dateS =
				// elem.select("p[class=info W_linkb W_textb]")
				// .select("a[date]").attr("date");
				String dateS = elem.select("p[class=info W_linkb W_textb]")
						.select("a[date]").attr("title");
				String url = elem.select("p[class=info W_linkb W_textb]")
						.select("a[date]").attr("href");
				String source = elem.select("p[class=info W_linkb W_textb]")
						.select("a[rel=nofollow]").text();

				logger.debug("KEYWORD：" + keyword + ",	CREATED_AT:" + dateS
						+ ", MID:" + id + ",	URL:" + url + ",	转发:"
						+ forwordCount + ",	收藏:" + colloctCount + ",	评论:"
						+ commentCount + ",	User:" + name + ",	SOURCE:"
						+ source + ",	TEXT:" + text);

				WeiboItem item = new WeiboItem();
				item.setColloctCount(colloctCount);
				item.setCommentCount(commentCount);
				item.setContent(text);
				item.setDate(dateS);
				item.setForwordCount(forwordCount);
				item.setCode(id);
				item.setKeyword(keyword);
				item.setName(name);
				item.setSource(source);
				item.setUrl(url);

				list.add(item);
			}

			itemDao.save(list);

		}
	}

	private List<Element> fetchPageElements(String keyword, String date,
			String page) throws IOException {
		Connection conn = Jsoup.connect("http://s.weibo.com/weibo/"
				+ policy.getKeyWord(keyword) + "&timescope=" + date
				+ "&nodup=1" + page);

		Map<String, String> header = new HashMap<String, String>();
		header.put("Host", "http://weibo.com");
		header.put(
				"User-Agent",
				"  Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_2) AppleWebKit/537.17 (KHTML, like Gecko) Chrome/26.0.1410.43 Safari/537.17");
		header.put("Accept",
				"  text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8");
		header.put("Accept-Language", "zh-cn,zh;q=0.5");
		header.put("Accept-Charset", "  GB2312,utf-8;q=0.7,*;q=0.7");
		header.put("Connection", "keep-alive");
		Document doc = conn.data(header).timeout(15000).get();

		String origin = doc.html();

		// 匹配文本块
		Matcher m = pattern.matcher(origin);

		while (m.find()) {

			String jsonStr = m.group();

			jsonStr = jsonStr.substring(jsonStr.indexOf("{"),
					jsonStr.lastIndexOf(")"));

			// 解析json,转换为实体类
			WeiboBlock block = JSON.parseObject(jsonStr, WeiboBlock.class);

			if (block.getHtml().trim()
					.startsWith("<div class=\"search_feed\">")) {

				doc = Jsoup.parse(block.getHtml());
			}
		}

		List<Element> elements = getAllElement(doc);
		int idle = 15000 + new Random().nextInt(5000);
		logger.debug("waiting " + idle / 1000 + " secends.");
		try {
			Thread.sleep(idle);
		} catch (InterruptedException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		}

		return elements;
	}

	/**
	 * 获取所有的结果正文节点
	 * 
	 * @param doc
	 * @return
	 */
	private List<Element> getAllElement(Document doc) {

		List<Element> resultList = new ArrayList<Element>();

		Elements elems = doc.select(".search_feed .feed_list");

		for (Element element : elems) {
			resultList.add(element);
		}

		return resultList;
	}
}

class KeyWordsPolicy {

	public String getKeyWord(String keyWordLine) {

		String keyWord = null;

		keyWordLine = keyWordLine.replaceAll("\"", "");
		keyWordLine = keyWordLine.replaceAll("AND", " ");
		keyWordLine = keyWordLine.replaceAll("OR", "|");
		// keyWordLine = keyWordLine.replaceAll("(", "");
		// keyWordLine = keyWordLine.replaceAll(")", "");

		if (keyWordLine.contains("|")) {

			String[] tempStrings = keyWordLine.split("|");
			if (tempStrings.length > 3) {
				for (int i = 0; i < 3; i++) {
					keyWord += tempStrings[i];
					keyWord += "|";
				}
			} else {
				keyWord = keyWordLine;
			}
		} else {
			keyWord = keyWordLine;
		}
		return java.net.URLEncoder.encode(java.net.URLEncoder.encode(keyWord));
	}

}
